package com.changxianggu.crawler.cloud.test;

import cn.hutool.core.collection.CollUtil;
import cn.hutool.core.map.MapUtil;
import cn.hutool.core.util.StrUtil;
import com.changxianggu.cloud.crawler.core.WebApplication;
import com.changxianggu.cloud.crawler.core.modules.storage.entity.CxgBook;
import com.changxianggu.cloud.crawler.core.modules.storage.service.ICxgBookService;
import com.changxianggu.cloud.crawler.core.utils.HttpUtil;
import com.changxianggu.cloud.crawler.core.utils.UserAgentUtil;
import lombok.extern.slf4j.Slf4j;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;

import java.math.BigDecimal;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

@RunWith(SpringRunner.class)
@SpringBootTest(classes = {WebApplication.class})// 指定启动类
@Slf4j
public class WenQuanShuJu {

    @Autowired
    ICxgBookService bookService;

    public static final String COOKIE = "JSESSIONID=87834587A86877E792A5C3F10C77056F; sso-cookie-key-dlib.cabplink.com=D3C8D39D656D57A9D4D77B0E0F85FD75BCF00E7C9423B047B595D060344C4B5B3E8A5CDB8E9BD32D; vjuids=-19c1d254b.170612b1ee4.0.90b2a0";
    public static final String ORIGIN = "http://dlib.cabplink.com";
    public static final String REFERER = "http://dlib.cabplink.com/search/1.shtml";
    public static final String URL_PRE = "http://dlib.cabplink.com";


    @Test
    public void initCrawler() throws Exception {
        doCrawler();
    }

    public void doCrawler() throws Exception {
        int pageNum = 1;
        int totalPage = 2765;
        int pageSize = 16;
        for (int i = pageNum; i <= totalPage; i++) {
            String url = "https://wqbook.wqxuetang.com/category/tid_0/pid_/pn_{}.html";
            String formatUrl = StrUtil.format(url, i);
            Map<String, Object> parameter = MapUtil.newHashMap(6);
            parameter.put("orderBy", 2);
            parameter.put("keyword", "978-7-112");
            parameter.put("pageSize", pageSize);
            parameter.put("searchType", 3);
            Map<String, Object> headers = MapUtil.newHashMap(5);
            headers.put("Cookie", COOKIE);
            headers.put("User-Agent", UserAgentUtil.random());
            headers.put("Origin", ORIGIN);
            headers.put("Referer", REFERER);
            String result = HttpUtil.doGet(formatUrl, headers, parameter);
            int tryCount = 1;
            while (result.contains("页面没有找到") && tryCount <= 5) {
                //睡眠2秒继续访问
                Thread.sleep(2000);
                result = HttpUtil.doPost(url, headers, parameter);
                tryCount++;
            }
            if (StrUtil.isNotBlank(result)) {
                Document htmlDoc = Jsoup.parse(result);
                Elements bookContentList = htmlDoc.select(".search-list .books_top_item");
                if (bookContentList != null && bookContentList.size() > 0) {
                    List<CxgBook> listBook = new ArrayList<>();
                    for (Element element : bookContentList) {
                        CxgBook bookEntity = new CxgBook();
                        Element imgElement = element.select(".top_item_img").first();
                        if (imgElement != null) {
                            Element imgEle = imgElement.select(".cate_item_img").first();
                            String href = imgElement.attr("href");
                            String title = imgElement.attr("title");
                            bookEntity.setDigitalUri(URL_PRE + href);
                            bookEntity.setBookName(title);
                            String srcStr = imgEle.attr("src");
                            String cover = URL_PRE + srcStr;
                            bookEntity.setCover(cover);
                        }
                        Elements detailElements = element.select(".cate_item_detail .item_detail_author");
                        Element authorEle = detailElements.get(0);
                        if (authorEle != null) {
                            bookEntity.setAuthor(authorEle.attr("title"));
                        }
                        Element pubDateEle = detailElements.get(2);
                        if (pubDateEle != null) {
                            String text = pubDateEle.text();
                            String[] strings = text.split("：");
                            if (strings.length == 2) {
                                String pubDateStr = strings[1] + " 00:00:00";
                                DateTimeFormatter dtf = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
                                LocalDateTime localDateTime = LocalDateTime.parse(pubDateStr, dtf);
                                bookEntity.setPubDate(localDateTime);
                            }
                        }
                        Element isbnEle = detailElements.get(3);
                        if (isbnEle != null) {
                            String[] strings = isbnEle.text().split("：");
                            if (strings.length == 2) {
                                String isbn = strings[1].replaceAll("-", "").replaceAll("—", "");
                                bookEntity.setBookIsbn(isbn);
                            }
                        }
                        Element pkEle = detailElements.get(4);
                        String pkId = null;
                        if (pkEle != null) {
                            String[] strings = pkEle.text().split("：");
                            if (strings.length == 2) {
                                pkId = strings[1];
                                bookEntity.setPkId(pkId);
                            }
                        }
                        Element priceElement = element.select(".cate_item_detail .item_detail_price .price_num").first();
                        if (priceElement != null) {
                            bookEntity.setPrice(BigDecimal.valueOf(Double.parseDouble(priceElement.text())));
                        }
                        CxgBook tmpBookEntity = getBookEntity(pkId);
                        if (tmpBookEntity != null) {
                            bookEntity.setPageNum(tmpBookEntity.getPageNum());
                            bookEntity.setBookVersion(tmpBookEntity.getBookVersion());
                            bookEntity.setBrief(tmpBookEntity.getBrief());
                            bookEntity.setCatalog(tmpBookEntity.getCatalog());
                        }
                        bookEntity.setIsSync(0);
                        bookEntity.setCreateTime(LocalDateTime.now());
                        bookEntity.setPressName("中国建筑工业出版社");
                        bookEntity.setStatus(1);
                        bookEntity.setPressId(51);
                        log.info(bookEntity.getBookName());
                        listBook.add(bookEntity);
                    }
                    bookService.saveBatch(listBook);
                    log.info("第" + i + "页数据保存成功");
                    log.warn("建工社--->" + i);
                }
            }
        }
    }

    public CxgBook getBookEntity(String numId) throws Exception {
        CxgBook bookEntity = null;
        if (StrUtil.isBlank(numId)) {
            return bookEntity;
        }
        String url = "http://dlib.cabplink.com/book/{}.shtml";
        Map<String, Object> headers = MapUtil.newHashMap(5);
        headers.put("Cookie", COOKIE);
        headers.put("User-Agent", UserAgentUtil.random());
        headers.put("Origin", ORIGIN);
        headers.put("Referer", REFERER);
        String formatUrl = StrUtil.format(url, numId);
        String result = HttpUtil.doGet(formatUrl, headers);
        int tryCount = 1;
        while (result.contains("页面没有找到") && tryCount <= 5) {
            //睡眠2秒继续访问
            Thread.sleep(2000);
            result = HttpUtil.doGet(formatUrl, headers);
            tryCount++;
        }
        if (StrUtil.isNotBlank(result)) {
            Document htmlDoc = Jsoup.parse(result);
            Elements detailList = htmlDoc.select(".book_detail .book_detail_more p");
            if (CollUtil.isNotEmpty(detailList)) {
                bookEntity = new CxgBook();
                for (Element element : detailList) {
                    String text = element.text();
                    if (StrUtil.isNotBlank(text) && text.startsWith("版次：")) {
                        String[] strings = text.split("：");
                        if (strings.length == 2) {
                            bookEntity.setBookVersion(strings[1]);
                        }
                    } else if (StrUtil.isNotBlank(text) && text.startsWith("页码：")) {
                        String[] strings = text.split("：");
                        if (strings.length == 2) {
                            bookEntity.setPageNum(Integer.parseInt(getNumbers(strings[1])));
                        }
                    }
                }
                Element briefEle = htmlDoc.select(".book_intro .book_intro_text").first();
                bookEntity.setBrief(briefEle.text());
                Element catalogEle = htmlDoc.select(".book_intro .book_intro_catalog").first();
                bookEntity.setCatalog(catalogEle.text());
            }
        }
        return bookEntity;
    }

    /**
     * 判断一个字符串是否都为数字
     *
     * @param strNum
     * @return
     */
    public boolean isDigit(String strNum) {
        Pattern pattern = Pattern.compile("[0-9]{1,}");
        Matcher matcher = pattern.matcher((CharSequence) strNum);
        return matcher.matches();
    }

    /**
     * 截取数字
     *
     * @param content
     * @return
     */
    public String getNumbers(String content) {
        Pattern pattern = Pattern.compile("\\d+");
        Matcher matcher = pattern.matcher(content);
        while (matcher.find()) {
            return matcher.group(0);
        }
        return "";
    }
}
